/* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "token.h"

#include <string.h>

#include "byteorder.h"
#include "wbyte.h"

/* ,
   CURRENT
   BEGIN
   END
    NEXTTOKEN
*/
#define SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION \
	(*source_offset_ptr) = p+1 ; \
	(*token_len) = (int32_t)( p+1 - (*token_begin) ) ; \

/* '''a'''
       CURRENT
      BEGIN
      END
          NEXTTOKEN
*/
#define SET_TOKEN_LESSTHAN_END_POSITION_AND_NEXTTOKEN_MOREMOREMORE_BEGIN_POSITION \
	(*source_offset_ptr) = p+3 ; \
	(*token_len) = (int32_t)( p - (*token_begin) ) ; \

/* "a"
     CURRENT
    BEGIN
    END
      NEXTTOKEN
*/
#define SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION \
	(*source_offset_ptr) = p+1 ; \
	(*token_len) = (int32_t)( p - (*token_begin) ) ; \

/* .M
    CURRENT
   BEGIN
   END
    NEXTTOKEN
*/
#define SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION \
	(*source_offset_ptr) = p ; \
	(*token_len) = (int32_t)( p - (*token_begin) ) ; \
	
/* 1.ToString()
     CURRENT
   BEGIN
   END
    NEXTTOKEN
*/
#define SET_TOKEN_LESSLESSTHAN_END_POSITION_AND_NEXTTOKEN_LESSLESSTHAN_BEGIN_POSITION \
	(*source_offset_ptr) = p-1 ; \
	(*token_len) = (int32_t)( p-1 - (*token_begin) ) ; \
	
enum TokenType TokenWord( uint16_t charset , char **source_offset_ptr , char *source_end , char **token_begin , int32_t *token_len , int32_t *source_row , int32_t *source_col )
{
	register char		*p = NULL ;
	enum TokenType		last_type ;
	
_GOTO_REDO_TOKEN :
	for( (*token_begin) = (*source_offset_ptr) ; (*token_begin) <= source_end ; (*token_begin)++ , (*source_col)++ )
	{
		if( *(*token_begin) == ' ' || *(*token_begin) == '\t' || *(*token_begin) == '\r' || *(*token_begin) == '\f' )
		{
			continue;
		}
		else if( *(*token_begin) == '\n' )
		{
			(*source_row)++;
			(*source_col) = 0 ;
			continue;
		}
		else
		{
			break;
		}
	}
	if( (*token_begin) > source_end )
		return TOKEN_TYPE_END_OF_SOURCE;
	
	p = (*token_begin) ;
	if( (*p) == ',' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_PART_OF_STATEMENT;
	}
	else if( (*p) == ';' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_END_OF_STATEMENT;
	}
	else if( (*p) == '.' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '.' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '.' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_ETC;
			}
			else
			{
				SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
				return TOKEN_TYPE_DOUBLE_DOT;
			}
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_MEMBER_OF;
		}
	}
	else if( (*p) == '"' )
	{
		(*source_col)++;
		for( p = (*token_begin) = p + 1 ; p <= source_end ; )
		{
			if( (unsigned char)(*p) >= 0x80 )
			{
				int len = EvaluateOneWbyteLength( charset , (unsigned char *)p , (unsigned char *)source_end ) ;
				if( len < 0 )
				{
					return TOKEN_TYPE_ERROR;
				}
				else if( len == 0 )
				{
					if( p == (*token_begin) )
						return TOKEN_TYPE_ERROR;
					
					break;
				}
				else
				{
					p += len ;
				}
			}
			else if( (*p) == '\\' )
			{
				p+=2;
				if( p > source_end )
					return TOKEN_TYPE_ERROR;
			}
			else if( (*p) == '"' )
			{
				break;
			}
			else
			{
				if( (*p) == '\n' )
				{
					(*source_row)++;
					(*source_col) = 0 ;
				}
				
				p++;
			}
		}
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		(*source_col)++;
		return TOKEN_TYPE_DOUBLE_QUOTES_STRING;
	}
	else if( (*p) == '\'' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '\'' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '\'' )
			{
				(*source_col) += 3 ;
				for( p = (*token_begin) = p + 1 ; p <= source_end ; )
				{
					if( (unsigned char)(*p) >= 0x80 )
					{
						int len = EvaluateOneWbyteLength( charset , (unsigned char *)p , (unsigned char *)source_end ) ;
						if( len < 0 )
						{
							return TOKEN_TYPE_ERROR;
						}
						else if( len == 0 )
						{
							if( p == (*token_begin) )
								return TOKEN_TYPE_ERROR;
							
							break;
						}
						else
						{
							p += len ;
						}
					}
					else if( (*p) == '\'' && ( p+1 <= source_end && *(p+1) == '\'' ) && ( p+2 <= source_end && *(p+2) == '\'' ) )
					{
						break;
					}
					else
					{
						if( (*p) == '\n' )
						{
							(*source_row)++;
							(*source_col) = 0 ;
						}
						
						p++;
					}
				}
				if( p > source_end )
					return TOKEN_TYPE_ERROR;
				
				SET_TOKEN_LESSTHAN_END_POSITION_AND_NEXTTOKEN_MOREMOREMORE_BEGIN_POSITION
				(*source_col) += 3;
				return TOKEN_TYPE_TRIPLE_QUOTES_STRING;
			}
			else
			{
				return TOKEN_TYPE_ERROR;
			}
		}
		else
		{
			return TOKEN_TYPE_ERROR;
		}
	}
	else if( (*p) == '`' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) != '`' )
		{
			(*source_col)++;
			for( p = (*token_begin) = p ; p <= source_end ; )
			{
				if( (unsigned char)(*p) >= 0x80 )
				{
					int len = EvaluateOneWbyteLength( charset , (unsigned char *)p , (unsigned char *)source_end ) ;
					if( len < 0 )
					{
						return TOKEN_TYPE_ERROR;
					}
					else if( len == 0 )
					{
						if( p == (*token_begin) )
							return TOKEN_TYPE_ERROR;
						
						break;
					}
					else
					{
						p += len ;
					}
				}
				else if( (*p) == '`' )
				{
					break;
				}
				else
				{
					if( (*p) == '\n' )
					{
						(*source_row)++;
						(*source_col) = 0 ;
					}
					
					p++;
				}
			}
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			(*source_col)++;
			return TOKEN_TYPE_IDENTIFICATION;
		}
		else
		{
			if( (*p) == '`' )
			{
				p++;
				if( p > source_end )
					return TOKEN_TYPE_ERROR;
				
				if( (*p) == '`' )
				{
					(*source_col) += 3 ;
					for( p = (*token_begin) = p + 1 ; p <= source_end ; )
					{
						if( (unsigned char)(*p) >= 0x80 )
						{
							int len = EvaluateOneWbyteLength( charset , (unsigned char *)p , (unsigned char *)source_end ) ;
							if( len < 0 )
							{
								return TOKEN_TYPE_ERROR;
							}
							else if( len == 0 )
							{
								if( p == (*token_begin) )
									return TOKEN_TYPE_ERROR;
								
								break;
							}
							else
							{
								p += len ;
							}
						}
						else if( (*p) == '`' && ( p+1 <= source_end && *(p+1) == '`' ) && ( p+2 <= source_end && *(p+2) == '`' ) )
						{
							break;
						}
						else
						{
							if( (*p) == '\n' )
							{
								(*source_row)++;
								(*source_col) = 0 ;
							}
							
							p++;
						}
					}
					if( p > source_end )
						return TOKEN_TYPE_ERROR;
					
					SET_TOKEN_LESSTHAN_END_POSITION_AND_NEXTTOKEN_MOREMOREMORE_BEGIN_POSITION
					(*source_col) += 3 ;
					return TOKEN_TYPE_TRIPLE_BACK_QUOTES_STRING;
				}
				else
				{
					return TOKEN_TYPE_ERROR;
				}
			}
			else
			{
				return TOKEN_TYPE_ERROR;
			}
		}
	}
	else if( (*p) == '+' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '+' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_PLUS_PLUS;
		}
		else if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_PLUS_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_PLUS;
		}
	}
	else if( (*p) == '-' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '-' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MINUS_MINUS;
		}
		else if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MINUS_EVAL;
		}
		/*
		else if( '0' <= (*p) && (*p) <= '9' )
		{
			goto _GOTO_TOKEN_NUMBER;
		}
		*/
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MINUS;
		}
	}
	else if( (*p) == '*' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MUL_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MUL;
		}
	}
	else if( (*p) == '/' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		(*source_col)++;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_DIV_EVAL;
		}
		else if( (*p) == '*' )
		{
			for( p++ , (*source_col)++ ; (*p) ; p++ , (*source_col)++ )
			{
				if( (*p) == '*' )
				{
					p++;
					if( p > source_end )
						return TOKEN_TYPE_ERROR;
					(*source_col)++;
					
					if( (*p) == '/' )
					{
						p++;
						break;
					}
				}
				else if( (*p) == '\n' )
				{
					(*source_row)++;
					(*source_col) = 0 ;
				}
			}
			
			(*source_offset_ptr) = p ;
			goto _GOTO_REDO_TOKEN;
		}
		else if( (*p) == '/' )
		{
			for( p++ , (*source_col)++ ; (*p) ; p++ , (*source_col)++ )
			{
				if( (*p) == '\n' )
				{
					p++;
					(*source_row)++;
					(*source_col) = 0 ;
					break;
				}
			}
			
			(*source_offset_ptr) = p ;
			goto _GOTO_REDO_TOKEN;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_DIV;
		}
	}
	else if( (*p) == '%' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MOD_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_OPERATOR_MOD;
		}
	}
	else if( (*p) == '=' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '&' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_REFER_EQUAL;
			}
			else if( (*p) == ':' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_ANCESTOR_EQUAL;
			}
			else
			{
				SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
				return TOKEN_TYPE_EQUAL;
			}
		}
		else if( (*p) == '*' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_EVAL;
		}
		else if( (*p) == '&' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_CAST_REFER;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_REFER;
		}
	}
	else if( (*p) == '<' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_COMPARE_LE;
		}
		else if( (*p) == '<' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '=' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_BIT_MOVELEFT_EVAL;
			}
			else
			{
				SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
				return TOKEN_TYPE_BIT_MOVELEFT;
			}
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_COMPARE_LT;
		}
	}
	else if( (*p) == '>' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_COMPARE_GE;
		}
		else if( (*p) == '>' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '=' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_BIT_MOVERIGHT_EVAL;
			}
			else
			{
				SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
				return TOKEN_TYPE_BIT_MOVERIGHT;
			}
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_COMPARE_GT;
		}
	}
	else if( (*p) == '&' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '&' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_LOGIC_AND;
		}
		else if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_AND_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_AND;
		}
	}
	else if( (*p) == '|' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '|' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_LOGIC_OR;
		}
		else if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_OR_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_OR;
		}
	}
	else if( (*p) == '^' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_XOR_EVAL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_BIT_XOR;
		}
	}
	else if( (*p) == '~' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_BIT_REVERSE;
	}
	else if( (*p) == '!' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '=' )
		{
			p++;
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
			
			if( (*p) == '&' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_REFER_NOT_EQUAL;
			}
			else if( (*p) == ':' )
			{
				SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
				return TOKEN_TYPE_ANCESTOR_NOT_EQUAL;
			}
			else
			{
				SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
				return TOKEN_TYPE_NOT_EQUAL;
			}
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_NOT;
		}
	}
	else if( (*p) == '(' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_BEGIN_OF_SUB_EXPRESSION;
	}
	else if( (*p) == ')' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_END_OF_SUB_EXPRESSION;
	}
	else if( (*p) == '[' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_BEGIN_OF_ARRAY;
	}
	else if( (*p) == ']' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_END_OF_ARRAY;
	}
	else if( (*p) == '{' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_BEGIN_OF_STATEMENT_SEGMENT;
	}
	else if( (*p) == '}' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_END_OF_STATEMENT_SEGMENT;
	}
	else if( (*p) == '?' )
	{
		p++;
		if( p > source_end )
			return TOKEN_TYPE_ERROR;
		
		if( (*p) == '.' )
		{
			SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
			return TOKEN_TYPE_MEMBER_OF_NOTNULL;
		}
		else
		{
			SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
			return TOKEN_TYPE_JUDGE;
		}
	}
	else if( (*p) == ':' )
	{
		SET_TOKEN_END_POSITION_AND_NEXTTOKEN_MORETHAN_BEGIN_POSITION
		return TOKEN_TYPE_ANOTHER_SIDE;
	}
	else if( '0' <= (*p) && (*p) <= '9' )
	{
/*
_GOTO_TOKEN_NUMBER :
*/
		last_type = TOKEN_TYPE_NUMBER ;
		
		p++;
		if( (*p) == 'x' )
		{
			p++;
			for( ; p <= source_end ; p++ )
			{
				if( ( '0' <= (*p) && (*p) <= '9' ) || ( 'a' <= (*p) && (*p) <= 'f' ) || ( 'A' <= (*p) && (*p) <= 'F' ) || (*p) == '_' )
				{
				}
				else
				{
					break;
				}
			}
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
		}
		else if( (*p) == 'o' )
		{
			p++;
			for( ; p <= source_end ; p++ )
			{
				if( ( '0' <= (*p) && (*p) <= '7' ) || (*p) == '_' )
				{
				}
				else
				{
					break;
				}
			}
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
		}
		else if( (*p) == 'b' )
		{
			p++;
			for( ; p <= source_end ; p++ )
			{
				if( (*p) == '0' || (*p) == '1' || (*p) == '_' )
				{
				}
				else
				{
					break;
				}
			}
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
		}
		else
		{
			for( ; p <= source_end ; p++ )
			{
				if( ( '0' <= (*p) && (*p) <= '9' ) || (*p) == '_' )
				{
				}
				else if( (*p) == '.' )
				{
					last_type = TOKEN_TYPE_FLOAT_NUMBER ;
				}
				else
				{
					if( *(p-1) == '.' )
					{
						SET_TOKEN_LESSLESSTHAN_END_POSITION_AND_NEXTTOKEN_LESSLESSTHAN_BEGIN_POSITION
						return TOKEN_TYPE_NUMBER;
					}
					
					break;
				}
			}
			if( p > source_end )
				return TOKEN_TYPE_ERROR;
		}
		
		SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
		return last_type;
	}
	else if( ( 'a' <= (*p) && (*p) <= 'z' ) || ( 'A' <= (*p) && (*p) <= 'Z' ) || (*p) == '_' || (unsigned char)(*p) >= 0x80 )
	{
		while( p <= source_end )
		{
			if( ( 'a' <= (*p) && (*p) <= 'z' ) || ( 'A' <= (*p) && (*p) <= 'Z' ) || ( '0' <= (*p) && (*p) <= '9' ) || (*p) == '_' )
			{
				p++;
			}
			else if( (unsigned char)(*p) >= 0x80 )
			{
				int len = EvaluateOneWbyteLength( charset , (unsigned char *)p , (unsigned char *)source_end ) ;
				if( len < 0 )
				{
					return TOKEN_TYPE_ERROR;
				}
				else if( len == 0 )
				{
					if( p == (*token_begin) )
						return TOKEN_TYPE_ERROR;
					
					break;
				}
				else
				{
					p += len ;
				}
			}
			else
			{
				break;
			}
		}
		
		SET_TOKEN_LESSTHAN_POSITION_AND_NEXTTOKEN_BEGIN_POSITION
		return TOKEN_TYPE_IDENTIFICATION;
	}
	else
	{
		return TOKEN_TYPE_ERROR;
	}
}

